using System;
using System.Text;
using System.IO;
using System.Text.RegularExpressions;

using Jeffrey.XFramework;
using Jeffrey.XFramework.XCrawl;
using Jeffrey.XFramework.XNet;

namespace Jeffrey.XProcess
{
    public class FetchHttp : Fetcher
    {
        private static readonly String[] CHARSET_EXTRACTOR = { @"<meta +[^>]*charset *= *(?<charset>[^'"" <>?]*)[^>]*>",
                                                               @"<\?xml +[^>]*encoding *=[ '""]*(?<charset>[^'"" <>?]*)[^?>]*\?>" };

        public FetchHttp(String name, String description, String classname, String path)
            : this(name, description, classname, path, null) { }
        public FetchHttp(String name, String description, String classname, String path, Filter filter)
            : base(name, description, classname, path, filter) { }

        protected override void Fetch(CrawlUri curi, ref XWebRequest xRequest)
        {
            XWebResponse xResponse = null;

            //Post request to the HTTP Sever
            xRequest = XWebRequest.Create(curi, xRequest, this.m_Controller);

            //Set HTTP-Headers
            xRequest.HttpHeaders["Accept"] = "*/*";
            xRequest.HttpHeaders["User-Agent"] = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 2.0.50727)";
            xRequest.Method = "GET";

            //Get the response from Sever
            xResponse = xRequest.GetResponse();

            if (curi.Status != CrawlUri.StatusCode.S_Succeed)
            {
                curi.Content = String.Empty;
                return;
            }

            //this return code must not include a message-body...
            if (curi.ResponseCode < 200  ||  //response code: 1xx(information) note: including the default value 0.
                curi.ResponseCode == 204 ||  //204 (no content)
                curi.ResponseCode == 304     //304 (not modified)
               )
            {
                /*
                 * do not need to get the content of the page...
                 * just return...
                 */
                return;
            }

            curi.Content = xResponse.GetContentString();

            //if error...
            if (curi.Status != CrawlUri.StatusCode.S_Succeed)
            {
                curi.Content = String.Empty;
                return;
            }
            else
            {
                /*
                 * Check for http-page redirection...
                 */
                if ( curi.ResponseCode == 201 ||
                     curi.ResponseCode == 301 ||
                     curi.ResponseCode == 302 ||
                     curi.ResponseCode == 303 ||
                     curi.ResponseCode == 307
                    )
                {
                    //clear the content and add the target uri as cuir's outlink
                    curi.Content = String.Empty;
                    curi.NormalLinks.Add(xResponse.RedirectedPage);
                }
                else
                {
                    if (xResponse.Encode.CodePage != CrawlOrder.PrimEncoding.CodePage)
                    {
                        //the charset has extracted in response-header...
                        curi.ContentEncoding = xResponse.Encode;
                    }
                    else
                    {
                        DetermineCharset(curi);

                        if (curi.ContentEncoding.CodePage != CrawlOrder.PrimEncoding.CodePage)
                        {
                            //ReEncoding opt.
                            Byte[] bytes = CrawlOrder.PrimEncoding.GetBytes(curi.Content);
                            curi.Content = curi.ContentEncoding.GetString(bytes);
                        }
                    }
                }
            }
            /*
             * reach here means that the page was gotten successfully...
             * So do not change the curi.Success flag, keep its value as this Processor
             * began to process it...
             */
        }

        private void DetermineCharset(CrawlUri curi)
        {
            String charset = String.Empty;

            try
            {
                for (Int32 i = 0; i < CHARSET_EXTRACTOR.Length; i++)
                {
                    Regex re = new Regex(CHARSET_EXTRACTOR[i], RegexOptions.IgnoreCase);
                    Match match = re.Match(curi.Content);
                    charset = match.Groups["charset"].Value;

                    if (charset != String.Empty)
                    {
                        switch (charset.ToLower())
                        {
                            case "utf8":  charset = "UTF-8";  break;
                            case "utf16": charset = "UTF-16"; break;
                            case "utf32": charset = "UTF-32"; break;
                            default: break;
                        }

                        curi.ContentEncoding = Encoding.GetEncoding(charset);
                        return;
                    }
                    else
                        curi.ContentEncoding = m_Controller.CrawlOrder.DefaultEncoding;
                }
            }
            catch (ArgumentException)
            {
                curi.ContentEncoding = m_Controller.CrawlOrder.DefaultEncoding;
            }
        }
    }
}
